### huseyin biyik 3.2012 huseyinbiyik*at,hotmail.com
import threading
import time
import datetime
import Queue
import ConfigParser
import sys

from kit import kit


def str2date(string):
	return datetime.datetime.strptime(str(string),'%Y-%m-%d %H:%M:%S')

def grab_single(kagit,url):
	global q
	toolkit=kit()
	browser=toolkit.create_browser()
	url=url.replace('[@@@hissekodu]',kagit)
	log=toolkit.browser_open(browser,url)
	lines=log.split('\n')
	if len(lines)> 0:
		array=[]
		diff=0
		hour=int(time.strftime('%H',time.gmtime()))
		if len(lines) > 300 and hour > 0 and hour < 12:
			diff=1
		today=(datetime.datetime.today() - datetime.timedelta(days=diff)).strftime('%Y-%m-%d')
		for line in lines:
			values=line.split(';')
			if len(values) == 3:
				try:
					full_time=datetime.datetime.strptime(today + ' ' + values[0] + ":00",'%Y-%m-%d %H:%M:%S')
				except:
#					print "python has a deadlock and i have a dirty workarond duh!"
					continue
				alis=float(values[1])
				array.append((kagit,full_time,alis))
	q.put(array)
	print "\n(!)" + kagit + " Sniffer Finished"
	return

def sql_eater(host,port,username,password,database):
	global q
	toolkit=kit()
	db=toolkit.connect_db(host,port,username,password,database)
	while True :
		if threading.activeCount() <= 4 and q.qsize()==0:
			break
		try:
			array=q.get(True,1)
		except:
			continue
		if len(array) < 1:
			continue
		today=str2date(array[0][1]).strftime('%Y-%m-%d')
		tomorrow=(str2date(array[0][1]) + datetime.timedelta(days=1)).strftime('%Y-%m-%d')
		cursor=db.cursor()
		cursor.execute("""select hisse,date,al from degerler where hisse=%s and date>%s and date<%s""", (array[0][0],today,tomorrow))
		table=cursor.fetchall()
		results=compare_arrays(table,array)
		cursor.executemany("""insert into degerler(hisse,date,al) values(%s,%s,%s)""", results)
		if len(results) > 0:
			print  str(len(results)) + " new points are added"
		cursor.close()
	db.close()
	return

def compare_arrays(references,newones):
	return list(set(newones)-set(references))

def compare_arrays2(references,newones):
	arr=[]
	for new in newones:
		eq=False
		for reference in references:
			if new[0]==reference[0] and new[1]==reference[1] and new[2]==reference[2]:
				eq=True
				break
		if not eq:
			arr.append(tuple(new))
	return arr
	
if __name__ == "__main__":

	global q
	starttime=float(time.time())

	try:
		cp= ConfigParser.RawConfigParser( )
		cp.read( r"hissedb.cfg" )
		sql_server=cp.get('SQL','server')
		sql_port=int(cp.get('SQL','port'))
		sql_username=cp.get('SQL','username')
		sql_password=cp.get('SQL','password')
		sql_database=cp.get('SQL','database')
		kagitlar=cp.get('SNIFFER','hisseler')
		kagitlar=kagitlar.split(',')
		url=cp.get('SNIFFER','url')
	except Exception,e:
		print e
		print 'No acceptable config file found'
		sys.exit()

	q = Queue.Queue()

	if len(sys.argv)>1:
		stock=sys.argv[1]
		print "Manually sniffing :" + stock 
		kagitlar=[stock]
	
	print "\n----------------------------------\n"
	print "updating database @" + sql_server
	print "total stocks to update: " + str(len(kagitlar))
	print "\n----------------------------------\n"

	threading.Thread(target=sql_eater,args=(sql_server,sql_port,sql_username,sql_password,sql_database)).start()
	threading.Thread(target=sql_eater,args=(sql_server,sql_port,sql_username,sql_password,sql_database)).start()
	threading.Thread(target=sql_eater,args=(sql_server,sql_port,sql_username,sql_password,sql_database)).start()

	for kagit in kagitlar:
		threading.Thread(target=grab_single,args=(kagit,url)).start()

	try:
		while True:
			if threading.activeCount() < 2:
				break
	except:
		print "Abnormally shut down"
	    
	bench=int((float(time.time())-starttime)*1000)
	print "\n\n      Performance"
	print "-----------------------"
	print "Total Time :" +str(bench) + "ms\nTime Per Stock : " + str(bench/len(kagitlar)) + "ms"
	print "-----------------------"